package org.rency.crawler.fetch.service.policy.impl;

import crawlercommons.robots.BaseRobotRules;
import crawlercommons.robots.SimpleRobotRulesParser;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URL;
import javax.annotation.Resource;
import org.apache.commons.lang.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.rency.common.utils.enums.CharsetEnum;
import org.rency.crawler.common.CrawlerConstants;
import org.rency.crawler.common.enums.PolicyState;
import org.rency.crawler.common.enums.SecLevelKind;
import org.rency.crawler.common.helper.ValidatorHelper;
import org.rency.crawler.domain.vo.MetaResourceDomain;
import org.rency.crawler.fetch.helper.FetchCarrier;
import org.rency.crawler.fetch.repository.MetaResourceRepository;
import org.rency.crawler.fetch.repository.RobotsPolicyRepository;
import org.rency.crawler.fetch.service.policy.PolicyService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;

/**
 * 描述: ${TODO}<br>
 * 最近修改时间:2017/11/22 11:55<br>
 *
 * @author CaiyuRen
 * @version V1.0
 * @since 2017/11/22 11:55
 */
@Service("robotsPolicyService")
public class RobotsPolicyServiceImpl implements PolicyService, CrawlerConstants {

  private static final Logger logger = LoggerFactory.getLogger(RobotsPolicyServiceImpl.class);

  @Resource
  private RobotsPolicyRepository robotsPolicyRepository;

  @Resource
  private MetaResourceRepository metaResourceRepository;

  @Override
  public boolean accept(FetchCarrier carrier) {
    boolean robotsSwitch = metaResourceRepository.getRobotsSwitch();
    if(!robotsSwitch){
      logger.debug("策略校验[{}]未打开,无需校验:{}.",this.getPolicyName(),carrier.getRequest().getFetchUri());
      return true;
    }
    boolean canAccess = false;
    String fetchNo = carrier.getRequest().getFetchNo();
    PolicyState state = robotsPolicyRepository.get(fetchNo);
    if(PolicyState.PASS == state){
      canAccess = true;
    }else if(PolicyState.REJECT == state){
      canAccess = false;
    }else{
      state = this.valid(carrier);
      robotsPolicyRepository.save(fetchNo,state);
    }
    return canAccess;
  }

  @Override
  public boolean retrySupport() {
    return false;
  }

  /**
   * 校验Robots策略
   * @param carrier
   * @return
   */
  private PolicyState valid(FetchCarrier carrier){
    String url = carrier.getRequest().getFetchUri();
    String hostName = carrier.getRequest().getHostAddr();
    if(StringUtils.isBlank(url) || StringUtils.isBlank(hostName)){
      logger.debug("策略校验{}-参数为空无需校验:{}.",this.getPolicyName(),url);
      return PolicyState.PASS;
    }
    String robotUrl = hostName+"/robots.txt";
    Document doc = null;
    try{
      MetaResourceDomain domain = metaResourceRepository.find(SecLevelKind.HTTP_CONNECTION_TIMEOUT);
      String timeout = DEFAULT_HTTP_CONNECT_TIMEOUT_MILL;
      if(domain != null){
        timeout = domain.getSecLevelValue();
      }
      doc = Jsoup.parse(new URL(robotUrl), Integer.parseInt(timeout));
    }catch(IOException e){
    }
    if(doc == null || StringUtils.isBlank(doc.text())){
      return PolicyState.PASS;
    }
    BaseRobotRules rules = null;
    try {
      MetaResourceDomain domain = metaResourceRepository.find(SecLevelKind.IDENTITY);
      ValidatorHelper.notNull(domain,SecLevelKind.IDENTITY.getMsg());
      rules = createRobotRules(hostName, domain.getSecLevelValue(), doc.text().getBytes(CharsetEnum.UTF8.getCode()));
    } catch (UnsupportedEncodingException e) {
    }
    if(rules == null){
      return PolicyState.PASS;
    }
    if(rules.isAllowed(url)){
      return PolicyState.PASS;
    }
    logger.info("Robots协议:域名[{}],[{}]目标禁止访问.",hostName,url);
    return PolicyState.REJECT;
  }

  private BaseRobotRules createRobotRules(String hostName,String crawlerName, byte[] content) {
    SimpleRobotRulesParser robotParser = new SimpleRobotRulesParser();
    return robotParser.parseContent(hostName, content, "text/plain", crawlerName);
  }

  @Override
  public String getPolicyName() {
    return "Robots协议";
  }

}
